
/**
 * @fileOverview Tokenizer
 * @author Xaephis <xaephis@gmail.com>
 * @version v 2.36 Tue Apr 26 2011 06:22:56 GMT+0800 (China Standard Time)
 */

Jx.declare('Jx.lib.Tokenizer', {

	strict: true,

	states: {

		'start': {}

	},

	stateCallback: {},

	callback: function(){},

	input: '',

	buffer: undefined,

	current: 0,

	lineIndexes: undefined,

	state: undefined,

	cache: undefined,

	lastToken: undefined,

	startState: 'start',

	garbageToken: '$garbage$',

	endToken: '$',

	lineFeedRe: /\n/,

	wildcard: '[*]',

	failed: false,

	completed: false,

	constructor: function(str, finalize){

		this.buffer = '';

		this.lineIndexes = [];

		this.cache = [];

		this.reset();

		this.$super();

	},

	defaultStateCallback: function(chr, input, state, previousState){

		this.appendBuffer(chr);

	},

	onAfterIterate: function(){

	},

	getStartState: function(){

		return this.startState;

	},

	getCurrentBuffer: function(){

		return this.buffer;

	},


	iterate: function(){

		var input = this.input,

			states = this.states,

			buffer = this.buffer,

			wildcard = this.wildcard,

			stateCallback = this.stateCallback || {},

			defaultStateCallback = this.defaultStateCallback,

			l = input.length,

			anchor = this.current,

			found, result,

			chr, c, o, prev, next;


		while((c = this.current) < l){

			prev = this.state;

			chr = input.charAt(c);

			if ( states.hasOwnProperty(prev) ){

				o = states[prev];

				next = o.hasOwnProperty(chr) ?

					o[chr]

					:

					o.hasOwnProperty(wildcard) ?

						o[wildcard]

						:

						null;

				if (next){

					this.state = next;

					result = (

						stateCallback.hasOwnProperty(next) ?

							stateCallback[next]

							:

							defaultStateCallback

					).call(this, chr, input, next, prev);

					if (result !== false){

						//-- save result
						if (Jx.isString(result, true)){

							found = {

								token: result,

								value: this.buffer,

								from: anchor,

								to: this.current

							}

						}

						//-- iterate if not backtracked
						this.current++;

						continue;

					}

				}

			}

			break;

		}

		this.onAfterIterate();

		return found;

	},

	appendBuffer: function(str){

		return this.buffer+=str;

	},

	backtrack: function(steps){

		var next;

		if (Jx.isNumber(steps)){

			next = this.current - steps;

			if (next >= -1 && next <= this.input.length){

				this.current = next;

			}

		}

		return this.current;

	},

	set: function(str, finalize){

		this.input = str;

		this.reset();

		if (finalize !== false){

			this.finalized = true;

		}

	},

	reset: function(){

		var indexes = this.lineIndexes,

			cache = this.cache;

		indexes.splice(0, indexes.length);

		cache.splice(0, cache.length);

		this.buffer = '';

		delete this.startState;

		this.state = this.startState;

		this.current = 0;

		delete this.failed;

		delete this.completed;

		delete this.lastToken;

	},


	next: function(){

		var buffer = this.buffer,

			cache = this.cache,

			anchor = this.current,

			start = anchor,

			garbage = start,

			old = this.state,

			l = this.input.length,

			found, end;



		if (cache.length){

			found = cache.shift();

		} else if (!this.failed && this.current < l){

			while(!found){

				//-- reset state
				this.state = this.getStartState();

				//-- clear buffer
				this.buffer = '';

				found = this.iterate();

				if (!found){

					garbage = this.current = ++start;

					if (this.current > l){

						if (this.finalized){

							this.failed = true;

						}

						break;

					}

				} else {

					this.current = found.to + 1;

				}

			}

			if (garbage > anchor){

				if (found){

					cache.push(found);

				}

				found = {

					token: this.garbageToken,

					value: this.input.substring(anchor, garbage),

					from: anchor,

					to: garbage

				};

			}


			//-- end
			if (this.finalized && this.current >= l){

				end = {

					token: this.endToken,

					value: '',

					from: l,

					to: l

				};

				if (found){

					cache.push(end);

				} else {

					found = end;

				}

			}

		}

		if (end){

			this.completed = true;

		}

		if (found){

			this.lastToken = found;

		}


		return !this.failed && found;

	},

	walk: function(callback, scope){

		scope = scope || Jx.isNumber(scope) ? scope : null;

		var ended = false, found;

		if (this.completed || this.failed){

			this.reset();

		}

		while(!ended){

			found = this.next();

			ended = !found;

			if (found){

				callback.call(scope, found.token, found.value, found.from, found.to + 1);

			}

		}

	},

	getCursorInfo: function(lineFeed){

		var current = this.current,

			input = this.input,

			line = 0,

			column = 0,

			str = input.substring(0, current.length);

		if (Jx.isNumber(lineFeed) && lineFeed > 0){

			lineFeed = String.fromCharCode(lineFeed);

		} else if (!Jx.isString(lineFeed)){

			lineFeed = '\n';

		}

		//-- find line
		for(var lastLineIndex, index = 0; index != -1;){

			line++;

			lastLineIndex = index;

			index = str.indexOf(lineFeed, index);

		}

		column = str.length - lastLineIndex;

		return {

			line: line,

			column: column

		};
		
	}




});
